package com.li.spark

import org.apache.spark.storage.StorageLevel
import org.apache.spark.{SparkConf, SparkContext}

/**
 * RDD持久化
 */
object PersistOp {
  def main(args: Array[String]): Unit = {
    val sc: SparkContext = getSparkContext

    val dataRDD = sc.textFile("hdfs://bigdata01:9000/hello_10000000.dat")
      .persist(StorageLevel.MEMORY_AND_DISK); //使用合适的持久化策略，RDD复用
    val start_time = System.currentTimeMillis()
    var count = dataRDD.count();
    println(count)
    val end_time = System.currentTimeMillis()
    println(end_time - start_time)


    val start_time2 = System.currentTimeMillis()
    var count2 = dataRDD.count();
    println(count2)
    val end_time2 = System.currentTimeMillis()
    println(end_time2 - start_time2)

    sc.stop();
  }


  private def getSparkContext = {
    //创建SparkContext
    val conf = new SparkConf();
    conf.setAppName("PersistOp").setMaster("local");
    new SparkContext(conf);
  }
}
